"""
Project Name : api_automation_framework
File Name : robot_txt_demo.py
Author : lldhsds
Date : 2024/12/28 11:53
Description: /robot.txt, 返回一个 robots.txt 格式的响应，该文件用于指示搜索引擎机器人可以或不可以抓取的页面。
"""

import requests

# GET: /robot.txt
# Returns some robots.txt rules.
# parameters: None
# response: 200(Robots file.)
def robot_txt_demo():
    url = "http://httpbin.org/robots.txt"
    headers = {'content-type': 'text/plain'}
    response = requests.get(url, headers=headers)
    print(response.status_code)
    print(response.text)

robot_txt_demo()
